import org.apache.commons.lang.StringUtils;
import utils.Trie;
import utils.StringsInit;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * @author tianjiashu
 * @Description     文章数量：434239  关键词：487175
 * @create 2023/2/21 16:29
 */
public class Main {

    private static final StringsInit STRINGS_INIT = new StringsInit();

    public static void main(String[] args) throws IOException, InterruptedException {
        int docNum = 20;
        /*
            构建二叉树
         */
        Trie trie = generateTrie();
        /*
            获取所有文章docId
         */
        List<String> docIdList = getDocIds();
        System.out.println("文章数量:"+docIdList.size());
//        docIdList = docIdList.subList(0,1000);
        long start = System.currentTimeMillis();
        /*
            批量获取并处理
         */
        System.out.println("批量获取并处理...");
        for (int i = 0; i < docIdList.size(); i+=docNum) {
            List<String> numDocIds = docIdList.subList(i, Math.min(i + docNum,docIdList.size()));
            String docIds = StringUtils.join(numDocIds, ",");
            Map<String,String> docBodyMap = getDocBody(docIds);

            docBodyMap.entrySet().parallelStream().forEach(entry->{
                trie.statistcal(entry.getKey(),entry.getValue());
            });
            if(i%1000 == 0){
                System.out.println("已处理文章数量："+i);
            }
        }
        long end = System.currentTimeMillis();
        long time = end - start;
        long l = TimeUnit.MILLISECONDS.toSeconds(time);
        System.out.println("共耗时："+l+"秒");
        System.out.println("处理完毕，保存结果...");
        trie.saveResult();
    }

    public static Map<String,String>  getDocBody(String docIds){
        Map<String,String> docBodyMap = new HashMap<>(20);
        String url = "elephant.hz.service.163.org/api/v1/elephant/doc/batch/retrieve.do?docids="+docIds+"&idc=cms&needPcBody=1";
        HttpRequest httpRequest = HttpRequest.get(url);
        httpRequest.setConnectionTimeout(3000);
        httpRequest.setReadTimeout(200000);
        HttpResponse response = httpRequest.execute();

        String body = response.body();
        JSONObject jsonObject = JSONObject.parseObject(body);
        JSONArray datas = jsonObject.getJSONArray("data");
        for (int i = 0; i < datas.size(); i++) {
            JSONObject data = datas.getJSONObject(i);
            String docBody = data.getString("body");
            String docId = data.getString("docid");
            docBody = docBody.replaceAll("</?[^>]+>", "");
            docBodyMap.put(docId,docBody);
        }
        return docBodyMap;
    }

    public static Trie generateTrie(){
        Map<String, String> stringMap = STRINGS_INIT.init("business.txt");
        System.out.println("简称数量:"+stringMap.size());
        Trie trie = new Trie();
        Set<Map.Entry<String, String>> entries = stringMap.entrySet();
        for (Map.Entry<String, String> entry : entries) {
            trie.insert(entry);
        }
        return trie;
    }

    public static List<String> getDocIds(){
        Map<String, String> stringMap = STRINGS_INIT.init("20230214-20230220_doc_id.txt");
        return new ArrayList<>(stringMap.keySet());
    }

}
